<?php 
	/*
	*生成随机的字符
	*/
	function createRandStr ()
	{	
		$s = '';
		$str = "0123456789abcdefghijklmnopqrstuvwxyz";   //   输出字符集  
		$n = 10;   //   输出串长度  
		$len = strlen($str)-1;
		for($i=0 ; $i<$n; $i++){
			$s .=  $str[rand(0,$len)];  
		}
		
		return $s;
	}
	/**
	 * 自动注册百合id
	 */
	function registerBaiheId()
	{
		//step1: get baihe view cookie
		$url = "http://love.baihe.com/Reg.action";
		$page = makeRequest($url,
						 "GET",
						 "",
						 true,
						 ROOT_PATH."/baihe/cookie/baihe_view_cookie.txt",
						 "",  
						 ""
						 );
		
		//step2: auto register
		$regInfo  = array();
		//$url = "http://love.baihe.com/PreRegister.action";
		$url = "http://love.baihe.com/NewPreRegister.action";
		$base = substr(strval(microtime(true)), -8);
		$parameter = array(
							"smzcStatus" => "",
							"smzcStatusSure" => 0,
							"regradio" => "",
							"goldRealInfos.cellphone" => "",
							"goldUser.email" => "usergirl_".$base."@126.com",
							"goldUser.userPass" => "123abc",
							"confirmPassword" => "123abc",
							"goldUser.loginId" => createRandStr(),
							"goldUser.gender" => "1",
							"dropMyCountry" => 86,
							"dropMyProvince" => 8611,
							"goldUser.cityCode" => 861105,
							"goldUserInfo.stature" => 170,
							"goldUserInfo.newdegree" => 5,
							"goldUserInfo.marriage" => 1,
							"goldUserInfo.newmonthlypay" => 4,
							"goldRealInfos.realname" => '长幼哪啦',
							"goldRealInfos.forcall" => '',
							"goldUser.know" => 0,
							"goldUserInfo.degree" => 2,
							"goldUserInfo.monthlypay" => 2,
							"goldUserInfo.housing" => 0,
							"goldUserInfo.minority" => 1,
							"goldUserInfo.profession" => 17,
							"goldUserInfo.religion" => 1,
							"goldUserInfo.smoke" => 1,
							"goldUserInfo.drink" => 1,
							"goldUserMatchInfo.opAge1" => 18,
							"goldUserMatchInfo.opAge2" => 25,
							"goldUserMatchInfo.opCity" => 8611,
							"goldUserMatchInfo.opStature1" => 150,
							"goldUserMatchInfo.opStature2" => 170,
							"goldUser.know" => "12",
							"goldUser.reservedStr" => "",
							"promit" => "on",
							"goldUserDesc.mobile" => '',
							"goldUserInfo.newhousing" => 1,
							"goldUserInfo.children" => 1,
							"channel" => '',
							"dropOpMarriage" => 2,
							"dropOpNewDegree" => 7,
							"dropOpNewMonthlyPay" => 11,
							"dropOpNewHousing" => 6,
							"dropOpChildren" => 3,
							"realInfoCheck" => 1,
							"goldUserDesc.userDesc" => '我来百合寻找我的另一半'.createRandStr(),
							"goldUserDesc.mobile" => '',
							"goldUserMatchInfo.opNewDegree" => 255,
							"goldUserMatchInfo.opNewMonthlyPay" => 4095,
							"goldUserMatchInfo.opNewHousing" => 127,
							"goldUserMatchInfo.opDegree" => 31,
							"goldUserMatchInfo.opMonthlypay" => 127,
							"goldUserMatchInfo.opHousing" => 63,
							"goldUser.birthday" => 19910404,
							"goldUserMatchInfo.opMarriage" => 7,
							"goldUserMatchInfo.opChildren" => 15,
							"goldUserMatchInfo.opMinority" => 0,
							"goldUserMatchInfo.opProfession" => 0,
							"goldUserMatchInfo.opReligion" => 0,
							"goldUserMatchInfo.opProvidefor" => 0,
							"goldUserMatchInfo.opSmoke" => 0,
							"goldUserMatchInfo.opDrink" => 0,
							"goldUserMatchInfo.mustStr" => '',
							);
		$opt_arr = array(CURLOPT_REFERER => "http://love.baihe.com/Reg.action",  //referer
			//			 CURLOPT_HEADER => true,						//输出中包含头信息
			//			 CURLOPT_COOKIE => $cookie,					
					);
		printlog("using:".$parameter['goldUser.email']." as register email", "register");
		$page = makeRequest($url,
						 "POST",
						 $parameter,
						 true, 
						 ROOT_PATH."/baihe/cookie/baihe_login_cookie.txt",  
						 ROOT_PATH."/baihe/cookie/baihe_view_cookie.txt",
						 $opt_arr
						 );
		//file_put_contents("test.html", $page);exit;
		//var_dump($page);			 
		//step3: parse uid from page
		preg_match_all('/<input id="userID".*? value="([0-9]+)/i', $page, $arr, PREG_SET_ORDER);
		return $arr[0][1];
	}
	
	//自动保存新注册的id
	function saveBaiheId($new_baihe_id)
	{
		global $conn;
		$today = date("Y-m-d", time());
		$now = date("Y-m-d H:i:s", time());
		$info = $now."\t".$new_baihe_id."\n";
		//保存至文件（防止数据库挂掉，备查）
		file_put_contents(ROOT_PATH."/baihe/log/register_log/$today.log", $info, FILE_APPEND);
		//保存至数据库
		$arr = array("site" => "baihe",
					"ymd" => $today,
					"register_time" => $now,
					"source_user_id" => $new_baihe_id
				);
		$sql = Sql::insert($arr, "register_log");
		$flag = $conn->query($sql);
		return $flag;
	}
	
	//获取某一天内百合最小的id
	function getBaiheMinId($ymd)
	{
		global $conn;
		$sql = "select min(source_user_id) from register_log where ymd='$ymd' and site='baihe'";
		$minId = $conn->result($sql);
		return $minId;
	}
	
	//获取某一天内百合最大的id
	function getBaiheMaxId($ymd)
	{
		global $conn;
		$sql = "select max(source_user_id) from register_log where ymd='$ymd' and site='baihe'";
		$maxId = $conn->result($sql);
		return $maxId;
	}
	
	//获取某一天内百合已经抓到的最大id
	function getBaiheMaxIdFinished($ymd)
	{
		global $conn;
		$tblName = getTblName($ymd);
		$sql = "select max(source_user_id) from $tblName where fetch_date='$ymd' and `from`='baihe'";//echo $sql;exit;
		$maxIdFinished = $conn->result($sql);
		return $maxIdFinished;
	}
	
	//获取百合某一个id的注册时间，debug时候用
	function getBaiheRegTime($uid)
	{
		global $conn;
		$sql = "select register_time from register_log where source_user_id='$uid' and site='baihe'";
		$regTime = $conn->result($sql);
		return $regTime;
	}
	
	//判断百合抓下来的页面是否包含正常的uid（通过经验判断，审核不通过的uid页面会非常小）
	function checkBaiheIdExists($page)
	{
		if(strlen($page)<130)
			return false;
		else
			return true;
	}
	
	//主函数： 根据起始id，终止id 和 ymd 这几个值来处理百合业务，包括抓取、提取和入库
	function baihe_crawl($startId, $endId, $ymd)
	{
		//创建page目录
		$page_dir = ROOT_PATH."/baihe/pages";
		if(!is_dir($page_dir)){
			mkdir($page_dir);
		}
		
		$page_dir = $page_dir."/".$ymd."/";
		if(!is_dir($page_dir)){
			mkdir($page_dir);
		}
		//如果抓取的速度不够，改用multi
		$number = 30;//concurrent request number
		$continueMiss = 0;
		$previousExists = true;
		$previousId = $startId;
		for ($uid=$startId; $uid<$endId; ){
			//保留本次开始的uid，万一抓取失败了重新抓取
			$previousId = $uid; 
			$perRequestFailedCount  = 0 ;//记录本次抓取失败的个数
			$handles = makeBaiheMultiRequests($uid, $number);
			foreach($handles as $thisuid => $handle){
				$page = curl_multi_getcontent($handle);
				if($page === false){ 
					printlog ("curl_multi_getcontent error: ".curl_error($handle), "crawl");
				}
				
				if(checkBaiheIdExists($page)){
					printlog("$thisuid available", "crawl");
					if(file_put_contents($page_dir.$thisuid.".html", $page)){	
						doWithBaihePage($thisuid, $ymd, $page);
					}
					else{
						printlog("save page error", "crawl");
					}
					$continueMiss = 0;
				}else{
					if(strlen($page)<30){
						echo "network problem ret is ".$page;
						$perRequestFailedCount++;
					}
					else{ 
						printlog("$thisuid not available", "crawl");
					}
					$continueMiss ++;
				}
				curl_close($handle);
				$uid++;
			}
				
			if($perRequestFailedCount>= $number/2){//如果有一半不存在，说明中途网络有问题了 
				$uid = $previousId;
				printlog("本次并发有 $perRequestFailedCount 个抓取失败，重新来。。。", "crawl");
				continue;//重新来
			}
			else{
				printlog("本次并发有 $perRequestFailedCount 个抓取失败，继续下一次抓取", "crawl");
			}
		}
	}
	//主函数： 根据起始id，终止id 和 ymd 这几个值来处理百合业务，包括抓取、提取和入库
	function baihe_crawlbak($startId, $endId, $ymd)
	{
		//创建page目录
		$page_dir = ROOT_PATH."/baihe/pages";
		if(!is_dir($page_dir)){
			mkdir($page_dir);
		}
		
		$page_dir = $page_dir."/".$ymd."/";
		if(!is_dir($page_dir)){
			mkdir($page_dir);
		}
		
		$continueMiss = 0; //连续id不存在的个数
		$cnt_succ = 0; //id存在的总个数
		for ($uid=$startId; $uid<$endId; $uid++){	
			$page = "";	
			//避免断线情况，这里采用循环抓取，直到抓到页面为止
			do{
				$url = "http://profile.baihe.com/new/BasicInfo.action?oppId=".$uid;
				//printlog("start crawl url : $url");
				$page = makeRequest($url, 
								"GET" , 
								"", 
								false, 
								"", 
								ROOT_PATH."/baihe/cookie/baihe_login_cookie.txt",
								"");
				//echo strlen($page)."\n"; 
			} while($page === false);
		
			if(checkBaiheIdExists($page)){
				printlog("$uid available", "crawl");
				if(file_put_contents($page_dir.$uid.".html", $page)){
					doWithBaihePage($uid, $ymd, $page);
				}
				else{
					printlog("save page error", "crawl");
				}
				$cnt_succ++;
				$continueMiss = 0;
			}else{
				printlog("$uid not available", "crawl");
				$continueMiss ++ ;
			}
	
			//防止出现id跳跃的现象，如果连续50个不存在，则以间隔50向前跳，直到抓取到一个id为止再恢复间隔为1
			if($continueMiss > 50){
				$uid += 50;
			}
		}
	} 
	/*
	*一下抓取多个页面
	*/
	function makeBaiheMultiRequests($uid, $number)
	{
		$mhandle = curl_multi_init();
		$handles = array();
		while($number>0){
			$url = "http://profile.baihe.com/new/BasicInfo.action?oppId=".$uid;
			$opt_arr = array(CURLOPT_REFERER => "http://search.baidu.com/search",  //referer	
					);
			$handles[$uid] = makeRequestResource($url,
												"GET" , 
												"", 
												false, 
												"", 
												ROOT_PATH."/baihe/cookie/baihe_login_cookie.txt", 
												$opt_arr
												);
			$uid++;
			$number--;
		}
		foreach($handles as $handle)
			curl_multi_add_handle($mhandle, $handle);
		//wait all request finish
		do {
			$status = curl_multi_exec($mhandle, $active);
		} while ($status === CURLM_CALL_MULTI_PERFORM || $active);
		curl_multi_close($mhandle);
		
		return $handles;
	}
	//生成百合登录后的cookie
	function makeBaiheCookie()
	{
		//step1: browser the mainpage and get jsession id ,dump it to baihe_portal_cookie.txt	
		makeRequest("http://www.baihe.com", 
					"GET", 
					"", 
					true, 
					ROOT_PATH."/baihe/cookie/baihe_portal_cookie.txt",
					"",
					""
					);
		//step2 login with the cookies which is fetched by previous request
		$parameter = array("_"=>time(), 
							"chkRememberMe"=>"Yes", 
							"time"=>time(), 
							"txtLoginEMail"=>"dh11111@126.com", 
							"txtLoginPwd"=>"123abc"
							);
		makeRequest("http://passport.baihe.com/loginForXs.jsp?",
					"GET", 
					$parameter, 
					true, 
					ROOT_PATH."/baihe/cookie/baihe_login_cookie.txt", 
					ROOT_PATH."/baihe/cookie/baihe_portal_cookie.txt",
					""
					);
	}
	function baihe_data($startId, $endId, $ymd)
	{
		if($startId >= $endId){
			echo 'finish';
			exit;
		}
		//创建page目录
		$page_dir = ROOT_PATH."/baihe/pages"."/".$ymd."/";
		if(!is_dir($page_dir)){
			echo 'ymd error ';
			//exit;
		}
		for ($uid=$startId; $uid<$endId; $uid++){
			$file = $page_dir.$uid.".html";
			$page = file_get_contents($file);
			doWithBaihePage($uid, $ymd, $page);
		}
		echo date('Y-m-d H:i:s'). 'end';
	}	
	//对一个抓下的百合页面的后续处理
	function doWithBaihePage($uid, $ymd, $page)
	{
		//check uid exists?
		global $conn, $global_province_list;
		$tblName = getTblName($ymd);
		$sql = "select * from $tblName where `from`='baihe' and source_user_id='$uid' limit 1";
		
		$exists = $conn->has_record($sql);
		if ($exists){
			printlog("$uid exists, do nothing", "crawl");
			return 1;
		}
		else{
			//extract
			$page = iconv("GBK", 'UTF-8//IGNORE', $page);
			$page = str_replace(array(" ","\t","\n","\r","&nbsp;"), "", $page);
			
			preg_match('/varmyName=(.*?)varmyCredit=/i', $page, $tem);
			if(isset($tem[1])){
				$temp = $tem[1];
			}else{
				$temp = '';
			}
			preg_match('/来自(.*?)岁/i', $page, $matches);
			if(isset($matches[1]))
				$cont = strip_tags($matches[1]);
			else{
				$cont = '';
			}
			//tmp: 山东济南34岁 或 江苏省20岁
			$province = "";
			foreach ($global_province_list as $prov){
				if (strpos($cont, $prov) !== false) {
					$province = $prov; //山东
					break;
				}
			}
			preg_match('/(\d\d)/i', $cont, $age);
			if (isset($age[1])) {
				$age = $age[1];
			}else{
				$age = 20;
			}
			
			$city = str_replace(array($province, $age, "省", "年龄：","岁",'：'), "", $cont); //济南
			
			//echo "age: $age province: $province city: $city tmp: $tmp ";exit;
			//$gender = simpleExtract('/oppGender=(\w+)/', $temp, "gender");
			preg_match('/oppGender=(\w+);/i', $temp, $gender);
			
			$gender = ($gender[1]=="true")?1:0; //男为1 女为0

			//$has_photo = simpleExtract('/oppMainPhoto=.*(nopic_.*)\./', $page, "has_photo");
			//$has_photo = ($has_photo)?1:0;
			preg_match('/oppHasPhotoNum=(\d+);/i', $temp, $photo);
			
			if (isset($photo[1]) && $photo[1]  != 0) {
				$has_photo =1;
			}else{
				$has_photo =0;
			}
			
			//insert
			if($province =='' && $city == '' && $has_photo ==0 && $age == 20){
				return 1;
			}
			$arr = array("fetch_date" => $ymd,
					"`from`" => "baihe",
					"source_user_id" => $uid,
					"age" => $age,
					"province" => $province,
					"city" => $city,
					"gender" => $gender,
					"has_photo" => $has_photo,
				);
		
			$sql = Sql::insert($arr, $tblName);
			$conn->query($sql);
		}
	}
	//对一个抓下的百合页面的后续处理
	function doWithBaihe($page)
	{
		//check uid exists?
		global $conn, $global_province_list;
		
			//extract
			$page = iconv("GBK", 'UTF-8//IGNORE', $page);
			//tmp: 山东济南34岁 或 江苏省20岁
			$tmp = simpleExtract('/来自(.*?)的/', $page, "tmp");
			$province = "";
			foreach ($global_province_list as $prov){
				if (strpos($tmp, $prov) !== false) {
					$province = $prov; //山东
					break;
				}
			}
			$age = simpleExtract('/(\d\d)/', $tmp, "age"); //20
			$city = str_replace(array($province, $age, "省", "岁"), "", $tmp); //济南
			//echo "age: $age province: $province city: $city tmp: $tmp ";exit;
			$gender = simpleExtract('/varoppGender=(\w+)/', $page, "gender");
			$gender = ($gender=="true")?1:0; //男为1 女为0
			//$has_photo = simpleExtract('/oppMainPhoto=.*(nopic_.*)\./', $page, "has_photo");
			//$has_photo = ($has_photo)?1:0;
			if (preg_match('/defaultUrl/i', $page)) {
				$has_photo =1;
			}else{
				$has_photo =0;
			}
			echo "has_photo:".$has_photo."\n";
			
			
	}
	
	

?>